{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"https://colab.research.google.com/github/timeseriesAI/tsai/blob/master/tutorial_nbs/05_TS_archs_comparison.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "created by Ignacio Oguiza - email: timeseriesAI@gmail.com"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import libraries 📚"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ## NOTE: UNCOMMENT AND RUN THIS CELL IF YOU NEED TO INSTALL/ UPGRADE TSAI\n",
    "# stable = False # True: stable version in pip, False: latest version from github\n",
    "# if stable: \n",
    "#     !pip install tsai -U >> /dev/null\n",
    "# else:      \n",
    "#     !pip install git+https://github.com/timeseriesAI/tsai.git -U >> /dev/null\n",
    "# ## NOTE: REMEMBER TO RESTART (NOT RECONNECT/ RESET) THE KERNEL/ RUNTIME ONCE THE INSTALLATION IS FINISHED"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tsai       : 0.2.3\n",
      "fastai     : 2.1.4\n",
      "fastcore   : 1.3.2\n",
      "torch      : 1.7.0+cu101\n"
     ]
    }
   ],
   "source": [
    "from tsai.all import *\n",
    "computer_setup()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiments 🧪"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I've run a small test to show how you can build any model in the `tsai` library using the `create_model` functions.\n",
    "\n",
    "The esiest way to do it is pass the architecture you want to use, a `TSDataLoaders` and any `kwargs` you'd like to use. The `create_model` function will pick up the necessary data from the `TSDataLoaders` object.\n",
    "\n",
    "I've used 2 multivariate time series datasets. As you can see, it's difficult to predict which architecture will work best for which dataset.\n",
    "\n",
    "Please, bear in mind that this is a very simple test without any hyperparameter tuning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>arch</th>\n",
       "      <th>hyperparams</th>\n",
       "      <th>total params</th>\n",
       "      <th>train loss</th>\n",
       "      <th>valid loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>OmniScaleCNN</td>\n",
       "      <td>{}</td>\n",
       "      <td>5239596</td>\n",
       "      <td>0.028580</td>\n",
       "      <td>0.069324</td>\n",
       "      <td>0.983333</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ResNet</td>\n",
       "      <td>{}</td>\n",
       "      <td>490758</td>\n",
       "      <td>0.025631</td>\n",
       "      <td>0.094521</td>\n",
       "      <td>0.972222</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LSTM_FCN</td>\n",
       "      <td>{}</td>\n",
       "      <td>347246</td>\n",
       "      <td>0.071825</td>\n",
       "      <td>0.100571</td>\n",
       "      <td>0.972222</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LSTM_FCN</td>\n",
       "      <td>{'shuffle': False}</td>\n",
       "      <td>336446</td>\n",
       "      <td>0.076918</td>\n",
       "      <td>0.107438</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FCN</td>\n",
       "      <td>{}</td>\n",
       "      <td>285446</td>\n",
       "      <td>0.070331</td>\n",
       "      <td>0.122366</td>\n",
       "      <td>0.955556</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>XceptionTime</td>\n",
       "      <td>{}</td>\n",
       "      <td>403420</td>\n",
       "      <td>0.395302</td>\n",
       "      <td>0.491704</td>\n",
       "      <td>0.955556</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>ResCNN</td>\n",
       "      <td>{}</td>\n",
       "      <td>268551</td>\n",
       "      <td>0.044637</td>\n",
       "      <td>0.148163</td>\n",
       "      <td>0.944444</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>InceptionTime</td>\n",
       "      <td>{}</td>\n",
       "      <td>460038</td>\n",
       "      <td>0.029208</td>\n",
       "      <td>0.152726</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>mWDN</td>\n",
       "      <td>{'levels': 4}</td>\n",
       "      <td>467038</td>\n",
       "      <td>0.030618</td>\n",
       "      <td>0.199803</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>xresnet1d34</td>\n",
       "      <td>{}</td>\n",
       "      <td>7232518</td>\n",
       "      <td>0.025482</td>\n",
       "      <td>0.406264</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 3, 'bidirectional': True}</td>\n",
       "      <td>585206</td>\n",
       "      <td>0.402990</td>\n",
       "      <td>0.311683</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 1, 'bidirectional': False}</td>\n",
       "      <td>51006</td>\n",
       "      <td>0.400282</td>\n",
       "      <td>0.350504</td>\n",
       "      <td>0.827778</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 2, 'bidirectional': True}</td>\n",
       "      <td>343606</td>\n",
       "      <td>0.360034</td>\n",
       "      <td>0.341538</td>\n",
       "      <td>0.822222</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 2, 'bidirectional': False}</td>\n",
       "      <td>131806</td>\n",
       "      <td>0.393701</td>\n",
       "      <td>0.394034</td>\n",
       "      <td>0.811111</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 3, 'bidirectional': False}</td>\n",
       "      <td>212606</td>\n",
       "      <td>0.446215</td>\n",
       "      <td>0.466275</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 1, 'bidirectional': True}</td>\n",
       "      <td>102006</td>\n",
       "      <td>0.435865</td>\n",
       "      <td>0.538259</td>\n",
       "      <td>0.761111</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             arch                              hyperparams  ...  accuracy  time\n",
       "0    OmniScaleCNN                                       {}  ...  0.983333    14\n",
       "1          ResNet                                       {}  ...  0.972222     7\n",
       "2        LSTM_FCN                                       {}  ...  0.972222     5\n",
       "3        LSTM_FCN                       {'shuffle': False}  ...  0.966667     6\n",
       "4             FCN                                       {}  ...  0.955556     5\n",
       "5    XceptionTime                                       {}  ...  0.955556    10\n",
       "6          ResCNN                                       {}  ...  0.944444     6\n",
       "7   InceptionTime                                       {}  ...  0.933333     9\n",
       "8            mWDN                            {'levels': 4}  ...  0.933333    10\n",
       "9     xresnet1d34                                       {}  ...  0.888889    15\n",
       "10           LSTM   {'n_layers': 3, 'bidirectional': True}  ...  0.833333     8\n",
       "11           LSTM  {'n_layers': 1, 'bidirectional': False}  ...  0.827778     5\n",
       "12           LSTM   {'n_layers': 2, 'bidirectional': True}  ...  0.822222     7\n",
       "13           LSTM  {'n_layers': 2, 'bidirectional': False}  ...  0.811111     5\n",
       "14           LSTM  {'n_layers': 3, 'bidirectional': False}  ...  0.777778     6\n",
       "15           LSTM   {'n_layers': 1, 'bidirectional': True}  ...  0.761111     5\n",
       "\n",
       "[16 rows x 7 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dsid = 'NATOPS' \n",
    "bs = 64\n",
    "X, y, splits = get_UCR_data(dsid, return_split=False)\n",
    "print(X.shape)\n",
    "tfms  = [None, [Categorize()]]\n",
    "dsets = TSDatasets(X, y, tfms=tfms, splits=splits)\n",
    "dls   = TSDataLoaders.from_dsets(dsets.train, dsets.valid, bs=[bs, bs*2])\n",
    "\n",
    "archs = [(FCN, {}), (ResNet, {}), (xresnet1d34, {}), (ResCNN, {}), \n",
    "         (LSTM, {'n_layers':1, 'bidirectional': False}), (LSTM, {'n_layers':2, 'bidirectional': False}), (LSTM, {'n_layers':3, 'bidirectional': False}), \n",
    "         (LSTM, {'n_layers':1, 'bidirectional': True}), (LSTM, {'n_layers':2, 'bidirectional': True}), (LSTM, {'n_layers':3, 'bidirectional': True}),\n",
    "         (LSTM_FCN, {}), (LSTM_FCN, {'shuffle': False}), (InceptionTime, {}), (XceptionTime, {}), (OmniScaleCNN, {}), (mWDN, {'levels': 4})]\n",
    "\n",
    "results = pd.DataFrame(columns=['arch', 'hyperparams', 'total params', 'train loss', 'valid loss', 'accuracy', 'time'])\n",
    "for i, (arch, k) in enumerate(archs):\n",
    "    model = create_model(arch, dls=dls, **k)\n",
    "    print(model.__class__.__name__)\n",
    "    learn = Learner(dls, model,  metrics=accuracy)\n",
    "    start = time.time()\n",
    "    learn.fit_one_cycle(100, 1e-3)\n",
    "    elapsed = time.time() - start\n",
    "    vals = learn.recorder.values[-1]\n",
    "    results.loc[i] = [arch.__name__, k, total_params(model)[0], vals[0], vals[1], vals[2], int(elapsed)]\n",
    "    results.sort_values(by='accuracy', ascending=False, ignore_index=True, inplace=True)\n",
    "    clear_output()\n",
    "    display(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>arch</th>\n",
       "      <th>hyperparams</th>\n",
       "      <th>total params</th>\n",
       "      <th>train loss</th>\n",
       "      <th>valid loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>XceptionTime</td>\n",
       "      <td>{}</td>\n",
       "      <td>401580</td>\n",
       "      <td>0.074442</td>\n",
       "      <td>1.488087</td>\n",
       "      <td>0.678832</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 3, 'bidirectional': True}</td>\n",
       "      <td>572414</td>\n",
       "      <td>0.000234</td>\n",
       "      <td>2.650948</td>\n",
       "      <td>0.669505</td>\n",
       "      <td>98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ResCNN</td>\n",
       "      <td>{}</td>\n",
       "      <td>260367</td>\n",
       "      <td>0.250915</td>\n",
       "      <td>1.266159</td>\n",
       "      <td>0.667883</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LSTM_FCN</td>\n",
       "      <td>{'shuffle': False}</td>\n",
       "      <td>314950</td>\n",
       "      <td>0.440930</td>\n",
       "      <td>1.236977</td>\n",
       "      <td>0.657340</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ResNet</td>\n",
       "      <td>{}</td>\n",
       "      <td>482574</td>\n",
       "      <td>0.030630</td>\n",
       "      <td>2.362593</td>\n",
       "      <td>0.650446</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 2, 'bidirectional': False}</td>\n",
       "      <td>125414</td>\n",
       "      <td>0.002532</td>\n",
       "      <td>2.354824</td>\n",
       "      <td>0.650041</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 2, 'bidirectional': True}</td>\n",
       "      <td>330814</td>\n",
       "      <td>0.000203</td>\n",
       "      <td>2.637241</td>\n",
       "      <td>0.644769</td>\n",
       "      <td>76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 3, 'bidirectional': False}</td>\n",
       "      <td>206214</td>\n",
       "      <td>0.002217</td>\n",
       "      <td>2.461025</td>\n",
       "      <td>0.633820</td>\n",
       "      <td>66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 1, 'bidirectional': False}</td>\n",
       "      <td>44614</td>\n",
       "      <td>0.054411</td>\n",
       "      <td>1.819723</td>\n",
       "      <td>0.616788</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 1, 'bidirectional': True}</td>\n",
       "      <td>89214</td>\n",
       "      <td>0.053924</td>\n",
       "      <td>1.760913</td>\n",
       "      <td>0.596918</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>LSTM_FCN</td>\n",
       "      <td>{}</td>\n",
       "      <td>326950</td>\n",
       "      <td>0.480608</td>\n",
       "      <td>1.561972</td>\n",
       "      <td>0.575020</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>OmniScaleCNN</td>\n",
       "      <td>{}</td>\n",
       "      <td>1432716</td>\n",
       "      <td>0.678315</td>\n",
       "      <td>2.100926</td>\n",
       "      <td>0.564477</td>\n",
       "      <td>93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>FCN</td>\n",
       "      <td>{}</td>\n",
       "      <td>270350</td>\n",
       "      <td>0.834407</td>\n",
       "      <td>1.705541</td>\n",
       "      <td>0.564071</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>mWDN</td>\n",
       "      <td>{'levels': 4}</td>\n",
       "      <td>461182</td>\n",
       "      <td>0.000195</td>\n",
       "      <td>2.706592</td>\n",
       "      <td>0.521492</td>\n",
       "      <td>120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>InceptionTime</td>\n",
       "      <td>{}</td>\n",
       "      <td>457614</td>\n",
       "      <td>0.253975</td>\n",
       "      <td>2.961505</td>\n",
       "      <td>0.448905</td>\n",
       "      <td>107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>xresnet1d34</td>\n",
       "      <td>{}</td>\n",
       "      <td>7234894</td>\n",
       "      <td>0.068992</td>\n",
       "      <td>4.470646</td>\n",
       "      <td>0.414436</td>\n",
       "      <td>181</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             arch                              hyperparams  ...  accuracy  time\n",
       "0    XceptionTime                                       {}  ...  0.678832   100\n",
       "1            LSTM   {'n_layers': 3, 'bidirectional': True}  ...  0.669505    98\n",
       "2          ResCNN                                       {}  ...  0.667883    60\n",
       "3        LSTM_FCN                       {'shuffle': False}  ...  0.657340    59\n",
       "4          ResNet                                       {}  ...  0.650446    78\n",
       "5            LSTM  {'n_layers': 2, 'bidirectional': False}  ...  0.650041    56\n",
       "6            LSTM   {'n_layers': 2, 'bidirectional': True}  ...  0.644769    76\n",
       "7            LSTM  {'n_layers': 3, 'bidirectional': False}  ...  0.633820    66\n",
       "8            LSTM  {'n_layers': 1, 'bidirectional': False}  ...  0.616788    45\n",
       "9            LSTM   {'n_layers': 1, 'bidirectional': True}  ...  0.596918    56\n",
       "10       LSTM_FCN                                       {}  ...  0.575020    53\n",
       "11   OmniScaleCNN                                       {}  ...  0.564477    93\n",
       "12            FCN                                       {}  ...  0.564071    46\n",
       "13           mWDN                            {'levels': 4}  ...  0.521492   120\n",
       "14  InceptionTime                                       {}  ...  0.448905   107\n",
       "15    xresnet1d34                                       {}  ...  0.414436   181\n",
       "\n",
       "[16 rows x 7 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dsid = 'LSST' \n",
    "bs = 64\n",
    "X, y, splits = get_UCR_data(dsid, return_split=False)\n",
    "print(X.shape)\n",
    "tfms  = [None, [Categorize()]]\n",
    "dsets = TSDatasets(X, y, tfms=tfms, splits=splits)\n",
    "dls   = TSDataLoaders.from_dsets(dsets.train, dsets.valid, bs=[bs, bs*2])\n",
    "\n",
    "archs = [(FCN, {}), (ResNet, {}), (xresnet1d34, {}), (ResCNN, {}), \n",
    "         (LSTM, {'n_layers':1, 'bidirectional': False}), (LSTM, {'n_layers':2, 'bidirectional': False}), (LSTM, {'n_layers':3, 'bidirectional': False}), \n",
    "         (LSTM, {'n_layers':1, 'bidirectional': True}), (LSTM, {'n_layers':2, 'bidirectional': True}), (LSTM, {'n_layers':3, 'bidirectional': True}),\n",
    "         (LSTM_FCN, {}), (LSTM_FCN, {'shuffle': False}), (InceptionTime, {}), (XceptionTime, {}), (OmniScaleCNN, {}), (mWDN, {'levels': 4})]\n",
    "\n",
    "results = pd.DataFrame(columns=['arch', 'hyperparams', 'total params', 'train loss', 'valid loss', 'accuracy', 'time'])\n",
    "for i, (arch, k) in enumerate(archs):\n",
    "    model = create_model(arch, dls=dls, **k)\n",
    "    print(model.__class__.__name__)\n",
    "    learn = Learner(dls, model,  metrics=accuracy)\n",
    "    start = time.time()\n",
    "    learn.fit_one_cycle(100, 1e-3)\n",
    "    elapsed = time.time() - start\n",
    "    vals = learn.recorder.values[-1]\n",
    "    results.loc[i] = [arch.__name__, k, total_params(model)[0], vals[0], vals[1], vals[2], int(elapsed)]\n",
    "    results.sort_values(by='accuracy', ascending=False, ignore_index=True, inplace=True)\n",
    "    clear_output()\n",
    "    display(results)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
